# coding=utf-8

import requests
import lxml.html
import lxml
import base64






import mongoengine

mongoengine.connect('test_my_zhihu_data')

class Zhihu_User_Profile(mongoengine.Document):
    user_name=mongoengine.StringField()
    user_be_agreed=mongoengine.StringField()
    user_be_thanked=mongoengine.StringField()
    user_followees=mongoengine.StringField()
    user_followers=mongoengine.StringField()
    user_education_school=mongoengine.StringField()
    user_education_subject=mongoengine.StringField()
    user_employment=mongoengine.StringField()
    user_employment_extra=mongoengine.StringField()
    user_location=mongoengine.StringField()
    user_gender=mongoengine.StringField()
    user_info=mongoengine.StringField()
    user_intro=mongoengine.StringField()
    user_url=mongoengine.StringField()




class Zhihu_Crawler():
    '''
        basic crawler
    '''

    def __init__(self, url, option="print_data_out"):
        '''
        init the crawler
        :param url:
        :param option:
        :return:
        '''

        self.option = option
        self.url = url
        self.header = {}
        self.header["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:43.0) Gecko/20100101 Firefox/43.0"
        self.header["Host"] = "www.zhihu.com"
        self.header["Referer"] = "www.zhihu.com"

        self.cookies={"z_c0":'"QUFBQWYzc2hBQUFYQUFBQVlRSlZUWTRHT0ZjN3g3QkpJeTYtampYTkhBSWFsS1N1WHhTOFpRPT0=|1460697487|60b9c14af82577221e3d221ba190b8199b540ddc"',
                "unlock_ticket":'QUFBQWYzc2hBQUFYQUFBQVlRSlZUWmFBRUZlT0pNWHhGOTVCektfQTZIVklQWG9pSVl5MDBnPT0=|1460697487|4fc8c06ce7f3396d21fdb8053b8b9f134ed07544',
                "login":'"MDUzOTdmZTFmMzMxNDdiNmFiZTMwMjExODc3NDQxMjU=|1460697486|eb6f2944a2b5abc1730dcec3d18ce6e2c33ac13d"',
                "l_n_c":"1",
                "q_c1":"77900a10afe74f899e80d6f09e8826fd|1460597791000|1457363816000",
                "l_cap_id":'"OTBmNzhlZTk0Mzc5NDQ2OGEwZGMzYWUwMzkzY2IzNzQ=|1460697465|5657c46ac9dcf30397429433679f027f36b2f928"',
                "d_c0":'"AIAAwMAipgmPTlDGjxmiBUMQNQHJfSlKHNY=|1458548004"',
                "cap_id":'"ZDE3NjIzZDE4ZjczNDk0Yjk2NzFiOWIwMzBlMjkwZjE=|1460697465|f3e8e3d47018e9af9864d9b8f82cb0339e65b454"'}


    def send_request(self):
        '''
        send a request to get HTML source

        '''
        added_followee_url=self.url+"/followees"
        try:
            r=requests.get(added_followee_url,cookies=self.cookies,headers=self.header,verify=False)
        except:
            re_crawl_url(self.url)
            return

        content=r.text


        if r.status_code==200:
            self.parse_user_profile(content)



    def process_xpath_source(self,source):
        if source:
            return source[0]
        else:
            return ''

    def parse_user_profile(self,html_source):
        '''
        parse the user's profile to mongo
        '''

        #initialize variances

        self.user_name=''
        self.fuser_gender=''
        self.user_location=''
        self.user_followees=''
        self.user_followers=''
        self.user_be_agreed=''
        self.user_be_thanked=''
        self.user_education_school=''
        self.user_education_subject=''
        self.user_employment=''
        self.user_employment_extra=''
        self.user_info=''
        self.user_intro=''

        tree= lxml.html.fromstring(html_source)

        #parse the html via lxml
        self.user_name=self.process_xpath_source(tree.xpath("//a[@class='name']/text()"))
        self.user_location=self.process_xpath_source(tree.xpath("//span[@class='location item']/@title"))
        self.user_gender=self.process_xpath_source(tree.xpath("//span[@class='item gender']/i/@class"))
        if "female" in self.user_gender and self.user_gender:
            self.user_gender="female"
        else:
            self.user_gender="male"
        self.user_employment=self.process_xpath_source(tree.xpath("//span[@class='employment item']/@title"))
        self.user_employment_extra=self.process_xpath_source(tree.xpath("//span[@class='position item']/@title"))
        self.user_education_school=self.process_xpath_source(tree.xpath("//span[@class='education item']/@title"))
        self.user_education_subject=self.process_xpath_source(tree.xpath("//span[@class='education-extra item']/@title"))
        try:
            self.user_followees=tree.xpath("//div[@class='zu-main-sidebar']//strong")[0].text
            self.user_followers=tree.xpath("//div[@class='zu-main-sidebar']//strong")[1].text
        except:
            return

        self.user_be_agreed=self.process_xpath_source(tree.xpath("//span[@class='zm-profile-header-user-agree']/strong/text()"))
        self.user_be_thanked=self.process_xpath_source(tree.xpath("//span[@class='zm-profile-header-user-thanks']/strong/text()"))
        self.user_info=self.process_xpath_source(tree.xpath("//span[@class='bio']/@title"))
        self.user_intro=self.process_xpath_source(tree.xpath("//span[@class='content']/text()"))

    def print_data_out(self):
        '''
        print out the user data
        '''

        print "*"*60
        print '用户名:%s\n' % self.user_name.encode('utf-8')
        print "用户性别:%s\n" % self.user_gender.encode('utf-8')
        print '用户地址:%s\n' % self.user_location.encode('utf-8')
        print "被同意:%s\n" % self.user_be_agreed.encode('utf-8')
        print "被感谢:%s\n" % self.user_be_thanked.encode('utf-8')
        print "被关注:%s\n" % self.user_followers.encode('utf-8')
        print "关注了:%s\n" % self.user_followees.encode('utf-8')
        print "工作:%s/%s" % (self.user_employment.encode('utf-8'),self.user_employment_extra.encode('utf-8'))
        print "教育:%s/%s" % (self.user_education_school.encode('utf-8'),self.user_education_subject.encode('utf-8'))
        print "用户信息:%s" % self.user_info.encode('utf-8')
        print "*"*60

    def store_data_to_mongo(self):
        '''
        store the data in mongo
        '''
        new_profile=Zhihu_User_Profile(
            user_name=self.user_name,
            user_be_agreed=self.user_be_agreed,
            user_be_thanked=self.user_be_thanked,
            user_followees=self.user_followees,
            user_followers=self.user_followers,
            user_education_school=self.user_education_school,
            user_education_subject=self.user_education_subject,
            user_employment=self.user_employment,
            user_employment_extra=self.user_employment_extra,
            user_location=self.user_location,
            user_gender=self.user_gender,
            user_info=self.user_info,
            user_intro=self.user_intro,
            user_url=self.url
        )
        new_profile.save()
        print "saved:%s \n" % self.user_name



if __name__ =='__main__':

    zhihu = Zhihu_Crawler('https://www.zhihu.com/people/liu-chang-xi-53')
    zhihu.send_request()
    zhihu.print_data_out()
    zhihu.store_data_to_mongo()














































